From: Jonathan Dieter Date: Fri, 20 Apr 2018 17:23:05 +0000 (+0300) Subject: Change file format to support streams and signatures X-Git-Tag: archive/raspbian/1.1.9+ds1-1+rpi1~1^2~307 X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/%22/%22http:/www.example.com/cgi/%22?a=commitdiff_plain;h=79d1d5a90050515d4f6100f56a25aa31695658eb;p=zchunk.git Change file format to support streams and signatures Signed-off-by: Jonathan Dieter --- diff --git a/src/lib/comp/comp.c b/src/lib/comp/comp.c index 6e61af8..f6aca3e 100644 --- a/src/lib/comp/comp.c +++ b/src/lib/comp/comp.c @@ -454,18 +454,29 @@ ssize_t comp_read(zckCtx *zck, char *dst, size_t dst_size, int use_dict) { /* End decompression chunk if we're on a chunk boundary */ if(zck->comp.data_idx == NULL) { zck->comp.data_idx = zck->index.first; + /* Skip first chunk if it's an empty dict */ + if(zck->comp.data_idx->comp_length == 0) + zck->comp.data_idx = zck->comp.data_idx->next; if(!zck_hash_init(&(zck->check_chunk_hash), &(zck->chunk_hash_type))) goto zck_hash_error; - zck->comp.data_loc = 0; + if(zck->comp.data_loc > 0) { + if(!zck_hash_update(&(zck->check_full_hash), zck->comp.data, + zck->comp.data_loc)) + goto zck_hash_error; + if(!zck_hash_update(&(zck->check_chunk_hash), zck->comp.data, + zck->comp.data_loc)) + goto zck_hash_error; + } + if(zck->comp.data_idx == NULL) { + free(src); + return 0; + } } if(zck->comp.data_loc == zck->comp.data_idx->comp_length) { if(comp_end_dchunk(zck, use_dict, zck->comp.data_idx->length) < 0) return -1; - if(zck->comp.data_idx == NULL) { - if(!zck_validate_file(zck)) - goto zck_hash_error; + if(zck->comp.data_idx == NULL) zck->comp.data_eof = True; - } continue; } diff --git a/src/lib/compint.c b/src/lib/compint.c index e0b4077..4ee6bd8 100644 --- a/src/lib/compint.c +++ b/src/lib/compint.c @@ -29,7 +29,7 @@ #include #include "zck_private.h" -void zck_compint_from_size(char *compint, size_t val, size_t *length) { +void compint_from_size(char *compint, size_t val, size_t *length) { for(unsigned char *i = (unsigned char *)compint; ; i++) { i[0] = val % 128; val = (val - i[0]) / 128; @@ -42,7 +42,7 @@ void zck_compint_from_size(char *compint, size_t val, size_t *length) { return; } -int zck_compint_to_size(size_t *val, const char *compint, size_t *length) { +int compint_to_size(size_t *val, const char *compint, size_t *length) { *val = 0; size_t old_val = 0; const unsigned char *i = (unsigned char *)compint; @@ -75,19 +75,19 @@ int zck_compint_to_size(size_t *val, const char *compint, size_t *length) { return True; } -int zck_compint_from_int(char *compint, int val, size_t *length) { +int compint_from_int(char *compint, int val, size_t *length) { if(val < 0) { zck_log(ZCK_LOG_ERROR, "Unable to compress negative integers\n"); return False; } - zck_compint_from_size(compint, (size_t)val, length); + compint_from_size(compint, (size_t)val, length); return True; } -int zck_compint_to_int(int *val, const char *compint, size_t *length) { +int compint_to_int(int *val, const char *compint, size_t *length) { size_t new = (size_t)*val; - if(!zck_compint_to_size(&new, compint, length)) + if(!compint_to_size(&new, compint, length)) return False; *val = (int)new; if(*val < 0) { diff --git a/src/lib/dl/dl.c b/src/lib/dl/dl.c index 365b764..fd36aa0 100644 --- a/src/lib/dl/dl.c +++ b/src/lib/dl/dl.c @@ -60,9 +60,8 @@ void zck_dl_free_dl_regex(zckDL *dl) { /* Write zeros to tgt->fd in location of tgt_idx */ int zck_dl_write_zero(zckCtx *tgt, zckIndexItem *tgt_idx) { char buf[BUF_SIZE] = {0}; - size_t tgt_data_offset = tgt->header_size + tgt->index_size; size_t to_read = tgt_idx->comp_length; - if(!seek_data(tgt->fd, tgt_data_offset + tgt_idx->start, SEEK_SET)) + if(!seek_data(tgt->fd, tgt->data_offset + tgt_idx->start, SEEK_SET)) return False; while(to_read > 0) { int rb = BUF_SIZE; @@ -158,10 +157,9 @@ int zck_dl_write_range(zckDL *dl, const char *at, size_t length) { &(dl->zck->chunk_hash_type))) return 0; dl->priv->write_in_chunk = idx->comp_length; - size_t offset = dl->zck->header_size + - dl->zck->index_size; - if(!seek_data(dl->dst_fd, offset + tgt_idx->start, - SEEK_SET)) + if(!seek_data(dl->dst_fd, + dl->zck->data_offset + tgt_idx->start, + SEEK_SET)) return 0; idx = NULL; tgt_idx = NULL; @@ -221,12 +219,10 @@ int zck_dl_write_and_verify(zckRange *info, zckCtx *src, zckCtx *tgt, zckIndexItem *src_idx, zckIndexItem *tgt_idx) { static char buf[BUF_SIZE] = {0}; - size_t src_data_offset = src->header_size + src->index_size; - size_t tgt_data_offset = tgt->header_size + tgt->index_size; size_t to_read = src_idx->comp_length; - if(!seek_data(src->fd, src_data_offset + src_idx->start, SEEK_SET)) + if(!seek_data(src->fd, src->data_offset + src_idx->start, SEEK_SET)) return False; - if(!seek_data(tgt->fd, tgt_data_offset + tgt_idx->start, SEEK_SET)) + if(!seek_data(tgt->fd, tgt->data_offset + tgt_idx->start, SEEK_SET)) return False; zckHash check_hash = {0}; if(!zck_hash_init(&check_hash, &(src->chunk_hash_type))) @@ -276,6 +272,7 @@ int PUBLIC zck_dl_copy_src_chunks(zckRange *info, zckCtx *src, zckCtx *tgt) { while(src_idx) { if(tgt_idx->comp_length == src_idx->comp_length && + tgt_idx->length == src_idx->length && memcmp(tgt_idx->digest, src_idx->digest, tgt_idx->digest_size) == 0) { found = True; @@ -454,7 +451,7 @@ int PUBLIC zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) { /* If we haven't downloaded enough for the index hash plus a few others, do * it now */ - if(!zck_dl_bytes(dl, url, zck->hash_type.digest_size+start+MAX_COMP_SIZE*2, + if(!zck_dl_bytes(dl, url, zck->hash_type.digest_size+start+MAX_COMP_SIZE*4, start, &buffer_len)) return False; /* Read and store the index hash */ @@ -477,14 +474,24 @@ int PUBLIC zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) { if(!zck_dl_bytes(dl, url, zck->index_size, start, &buffer_len)) return False; + if(!zck_header_hash(zck)) + return False; if(!zck_read_index(zck)) return False; + /* Read signatures */ + if(!zck_read_sig(zck)) + return False; + if(!close_read_header(zck)) + return False; + if(!zck_validate_header(zck)) + return False; + /* Write zeros to rest of file */ zckIndex *info = &(dl->info.index); info->hash_type = zck->index.hash_type; zck_log(ZCK_LOG_DEBUG, "Writing zeros to rest of file: %llu\n", zck->index.length + zck->index_size + start); - if(!zck_zero_bytes(dl, zck->index.length, zck->header_size + zck->index_size, &buffer_len)) + if(!zck_zero_bytes(dl, zck->index.length, zck->data_offset, &buffer_len)) return False; return True; } diff --git a/src/lib/dl/range.c b/src/lib/dl/range.c index 4d12669..4e0e44d 100644 --- a/src/lib/dl/range.c +++ b/src/lib/dl/range.c @@ -209,7 +209,6 @@ char *zck_range_get_char(zckRangeItem **range, int max_ranges) { buf_size = (int)(buf_size * 1.5); output = realloc(output, buf_size); if(output == NULL) { - free(output); zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", buf_size); return NULL; diff --git a/src/lib/hash/hash.c b/src/lib/hash/hash.c index 1fb7f6f..3d94600 100644 --- a/src/lib/hash/hash.c +++ b/src/lib/hash/hash.c @@ -135,7 +135,7 @@ void zck_hash_close(zckHash *hash) { /* Returns 1 if data hash matches, 0 if it doesn't and -1 if failure */ int PUBLIC zck_hash_check_data(zckCtx *zck, int dst_fd) { - if(!seek_data(dst_fd, zck->header_size + zck->index_size, SEEK_SET)) + if(!seek_data(dst_fd, zck->data_offset, SEEK_SET)) return -1; if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type))) return -1; diff --git a/src/lib/header.c b/src/lib/header.c index 4344486..7e158fa 100644 --- a/src/lib/header.c +++ b/src/lib/header.c @@ -52,22 +52,56 @@ return False; \ } -int zck_read_initial(zckCtx *zck) { - VALIDATE_READ(zck); +int check_flags(zckCtx *zck, char *header, size_t *length) { + zck->has_streams = header[8] & 0x01; + if(zck->has_streams) + zck_log(ZCK_LOG_INFO, "Archive has streams\n"); + if((header[8] & 0xfe) != 0 || header[7] != 0 || header[6] != 0 || + header[5] != 0) { + zck_log(ZCK_LOG_ERROR, "Unknown flags(s) set\n"); + return False; + } + *length += 4; + return True; +} - char *header = zmalloc(5 + MAX_COMP_SIZE); - size_t length = 0; - if(header == NULL) { +int add_to_header_string(zckCtx *zck, char *data, size_t length) { + VALIDATE(zck); + + zck->header_string = realloc(zck->header_string, zck->header_size + length); + if(zck->header_string == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", - zck->hash_type.digest_size); + zck->header_size + length); return False; } + memcpy(zck->header_string + zck->header_size, data, length); + zck->header_size += length; + return True; +} - zck_log(ZCK_LOG_DEBUG, "Reading magic and hash type\n"); - if(!read_data(zck->fd, header, 5 + MAX_COMP_SIZE)) { - free(header); +int add_to_sig_string(zckCtx *zck, char *data, size_t length) { + VALIDATE(zck); + + zck->sig_string = realloc(zck->sig_string, zck->sig_size + length); + if(zck->sig_string == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + zck->sig_size + length); return False; } + memcpy(zck->sig_string + zck->sig_size, data, length); + zck->sig_size += length; + return True; +} + +int zck_read_initial(zckCtx *zck) { + VALIDATE_READ(zck); + + char *header = NULL; + size_t length = 0; + + zck_log(ZCK_LOG_DEBUG, "Reading magic, flags and hash type\n"); + if(read_header(zck, &header, 9 + MAX_COMP_SIZE) < 9 + MAX_COMP_SIZE) + return False; if(memcmp(header, "\0ZCK1", 5) != 0) { free(header); @@ -77,18 +111,19 @@ int zck_read_initial(zckCtx *zck) { } length += 5; + if(!check_flags(zck, header, &length)) + return False; int hash_type = 0; - if(!zck_compint_to_int(&hash_type, header+length, &length)) + if(!compint_to_int(&hash_type, header+length, &length)) return False; if(!zck_hash_setup(&(zck->hash_type), hash_type)) return False; - if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type))) - return False; - if(!seek_data(zck->fd, length, SEEK_SET)) + + /* Return any unused bytes from read_header */ + if(!read_header_unread(zck, 9 + MAX_COMP_SIZE - length)) return False; - zck->header_string = header; - zck->header_size = length; - return True; + + return add_to_header_string(zck, header, length); } int zck_read_header_hash(zckCtx *zck) { @@ -99,35 +134,23 @@ int zck_read_header_hash(zckCtx *zck) { "Reading index hash before initial bytes are read\n"); return False; } - size_t length = zck->header_size; - char *header = zck->header_string; - zck->header_string = NULL; - zck->header_size = 0; - header = realloc(header, length + zck->hash_type.digest_size); - if(header == NULL) { - zck_log(ZCK_LOG_ERROR, "Unable to reallocate %lu bytes\n", - length + zck->hash_type.digest_size); - return False; - } + + char *header = NULL; + char *digest = zmalloc(zck->hash_type.digest_size); if(digest == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", zck->hash_type.digest_size); return False; } - zck_log(ZCK_LOG_DEBUG, "Reading index hash\n"); - if(!read_data(zck->fd, digest, zck->hash_type.digest_size)) { + zck_log(ZCK_LOG_DEBUG, "Reading header hash\n"); + if(read_header(zck, &header, zck->hash_type.digest_size) + < zck->hash_type.digest_size) { free(digest); - free(header); return False; } - - /* Set hash to zeros in header string so we can validate it later */ - memset(header + length, 0, zck->hash_type.digest_size); - length += zck->hash_type.digest_size; - zck->index_digest = digest; - zck->header_string = header; - zck->header_size = length; + memcpy(digest, header, zck->hash_type.digest_size); + zck->header_digest = digest; return True; } @@ -139,25 +162,18 @@ int zck_read_ct_is(zckCtx *zck) { "Reading compression type before hash type is read\n"); return False; } - size_t length = zck->header_size; - char *header = zck->header_string; - zck->header_string = NULL; - zck->header_size = 0; - header = realloc(header, length + MAX_COMP_SIZE*2); - if(header == NULL) { - zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", - length + MAX_COMP_SIZE); - return False; - } + char *header = NULL; + size_t length = 0; + zck_log(ZCK_LOG_DEBUG, "Reading compression type and index size\n"); - if(!read_data(zck->fd, header + length, MAX_COMP_SIZE*2)) + if(read_header(zck, &header, MAX_COMP_SIZE*2) < MAX_COMP_SIZE*2) return False; int tmp = 0; /* Read and initialize compression type */ - if(!zck_compint_to_int(&tmp, header + length, &length)) + if(!compint_to_int(&tmp, header, &length)) return False; if(!zck_set_ioption(zck, ZCK_COMP_TYPE, tmp)) return False; @@ -165,37 +181,78 @@ int zck_read_ct_is(zckCtx *zck) { return False; /* Read and initialize index size */ - if(!zck_compint_to_int(&tmp, header + length, &length)) + if(!compint_to_int(&tmp, header + length, &length)) return False; zck->index_size = tmp; - if(!seek_data(zck->fd, length, SEEK_SET)) + /* Return any unused bytes from read_header */ + if(!read_header_unread(zck, MAX_COMP_SIZE*2 - length)) + return False; + + return add_to_header_string(zck, header, length); +} + +int zck_header_hash(zckCtx *zck) { + /* Calculate checksum to this point */ + if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type))) + return False; + if(!zck_hash_update(&(zck->check_full_hash), zck->header_string, + zck->header_size)) return False; - zck->header_string = header; - zck->header_size = length; return True; } int zck_read_index(zckCtx *zck) { VALIDATE_READ(zck); - char *index = zmalloc(zck->index_size); - if(!index) { - zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", - zck->index_size); - return False; - } + char *header = NULL; zck_log(ZCK_LOG_DEBUG, "Reading index\n"); - if(!read_data(zck->fd, index, zck->index_size)) { - free(index); + if(!read_header(zck, &header, zck->index_size)) + return False; + + if(!zck_index_read(zck, header, zck->index_size)) + return False; + + return True; +} + +int zck_read_sig(zckCtx *zck) { + VALIDATE_READ(zck); + + if(zck->header_string == NULL) { + zck_log(ZCK_LOG_ERROR, + "Reading signatures before hash type is read\n"); return False; } - if(!zck_index_read(zck, index, zck->index_size)) { - free(index); + + char *header = NULL; + size_t length = 0; + + /* Get signature size */ + ssize_t rd = read_header(zck, &header, MAX_COMP_SIZE); + if(rd < 0) + return False; + + if(!compint_to_int(&(zck->sigs.count), header, &length)) + return False; + + /* We don't actually support signatures yet, so bail if there is one */ + zck_log(ZCK_LOG_DEBUG, "Signature count: %i\n", zck->sigs.count); + if(zck->sigs.count > 0) { + zck_log(ZCK_LOG_ERROR, "Signatures aren't supported yet\n"); return False; } - free(index); - return True; + + if(!zck_hash_update(&(zck->check_full_hash), header, + length)) + return False; + + /* Return any unused bytes from read_header */ + if(!read_header_unread(zck, rd - length)) + return False; + + zck->data_offset = zck->hdr_buf_size; + return add_to_sig_string(zck, header, length); } int zck_read_header(zckCtx *zck) { @@ -207,55 +264,116 @@ int zck_read_header(zckCtx *zck) { return False; if(!zck_read_ct_is(zck)) return False; + if(!zck_header_hash(zck)) + return False; if(!zck_read_index(zck)) return False; + if(!zck_read_sig(zck)) + return False; + if(!close_read_header(zck)) + return False; + if(!zck_validate_header(zck)) + return False; if(!zck_import_dict(zck)) return False; return True; } int zck_header_create(zckCtx *zck) { - int header_malloc = 5 + MAX_COMP_SIZE + zck->hash_type.digest_size + + int header_malloc = 9 + MAX_COMP_SIZE + zck->hash_type.digest_size + MAX_COMP_SIZE*2; + char *header = zmalloc(header_malloc); if(header == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", header_malloc); return False; } + size_t start = 0; size_t length = 0; - memcpy(header+length, "\0ZCK1", 5); + memcpy(header, "\0ZCK1", 5); length += 5; - zck_compint_from_size(header+length, zck->hash_type.type, &length); + /* First three bytes of flags are always 0 */ + length += 3; + /* Final byte for flags */ + if(zck->has_streams) + header[length] &= 1; + length += 1; + compint_from_size(header+length, zck->hash_type.type, &length); + if(!add_to_header_string(zck, header, length)) { + free(header); + return False; + } + start = length; /* If we have the digest, write it in, otherwise write zeros */ - if(zck->index_digest) - memcpy(header+length, zck->index_digest, zck->hash_type.digest_size); + if(zck->header_digest) + memcpy(header+length, zck->header_digest, zck->hash_type.digest_size); else memset(header+length, 0, zck->hash_type.digest_size); length += zck->hash_type.digest_size; + start = length; - if(!zck_compint_from_int(header+length, zck->comp.type, &length)) { + /* Write out compression type and index size */ + if(!compint_from_int(header+length, zck->comp.type, &length)) { + free(header); + return False; + } + compint_from_size(header+length, zck->index_size, &length); + if(!add_to_header_string(zck, header+start, length-start)) { free(header); return False; } - zck_compint_from_size(header+length, zck->index_size, &length); + start = length; + + /* Shrink header to actual size */ header = realloc(header, length); if(header == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to reallocate %lu bytes\n", length); return False; } - if(zck->header_string) - free(zck->header_string); - zck->header_string = header; - zck->header_size = length; + if(zck->hdr_buf) + free(zck->hdr_buf); + zck->hdr_buf = header; + zck->hdr_buf_size = length; + return True; +} + +int zck_sig_create(zckCtx *zck) { + char *header = zmalloc(MAX_COMP_SIZE); + if(header == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", MAX_COMP_SIZE); + return False; + } + size_t length = 0; + + zck_log(ZCK_LOG_DEBUG, "Calculating %i signatures\n", zck->sigs.count); + + /* Write out signature count and signatures */ + if(!compint_from_int(header+length, zck->sigs.count, &length)) { + free(header); + return False; + } + for(int i=0; isigs.count; i++) { + // TODO: Add signatures + } + zck->sig_string = header; + zck->sig_size = length; return True; } int zck_write_header(zckCtx *zck) { VALIDATE_WRITE(zck); - if(!write_data(zck->fd, zck->header_string, zck->header_size)) + if(!write_data(zck->fd, zck->hdr_buf, zck->hdr_buf_size)) return False; return True; } + +int zck_write_sigs(zckCtx *zck) { + VALIDATE_WRITE(zck); + + if(!write_data(zck->fd, zck->sig_string, zck->sig_size)) + return False; + return True; +} diff --git a/src/lib/index/index_common.c b/src/lib/index/index_common.c index 396c6c1..0fc081e 100644 --- a/src/lib/index/index_common.c +++ b/src/lib/index/index_common.c @@ -88,8 +88,20 @@ void zck_index_free(zckCtx *zck) { free(zck->header_string); zck->header_string = NULL; } - if(zck->index_digest) { - free(zck->index_digest); - zck->index_digest = NULL; + zck->header_size = 0; + if(zck->sig_string) { + free(zck->sig_string); + zck->sig_string = NULL; + } + zck->sig_size = 0; + if(zck->hdr_buf) { + free(zck->hdr_buf); + zck->hdr_buf = NULL; + } + zck->hdr_buf_read = 0; + zck->hdr_buf_size = 0; + if(zck->header_digest) { + free(zck->header_digest); + zck->header_digest = NULL; } } diff --git a/src/lib/index/index_create.c b/src/lib/index/index_create.c index 28d8779..a09fe97 100644 --- a/src/lib/index/index_create.c +++ b/src/lib/index/index_create.c @@ -64,8 +64,8 @@ int zck_index_finalize(zckCtx *zck) { /* Write index */ index = zmalloc(index_malloc); - zck_compint_from_size(index+index_size, zck->index.hash_type, &index_size); - zck_compint_from_size(index+index_size, zck->index.count, &index_size); + compint_from_size(index+index_size, zck->index.hash_type, &index_size); + compint_from_size(index+index_size, zck->index.count, &index_size); memcpy(index+index_size, zck->full_hash_digest, zck->hash_type.digest_size); index_size += zck->hash_type.digest_size; if(zck->index.first) { @@ -75,10 +75,10 @@ int zck_index_finalize(zckCtx *zck) { memcpy(index+index_size, tmp->digest, zck->index.digest_size); index_size += zck->index.digest_size; /* Write compressed size */ - zck_compint_from_size(index+index_size, tmp->comp_length, + compint_from_size(index+index_size, tmp->comp_length, &index_size); /* Write uncompressed size */ - zck_compint_from_size(index+index_size, tmp->length, &index_size); + compint_from_size(index+index_size, tmp->length, &index_size); tmp = tmp->next; } @@ -92,14 +92,18 @@ int zck_index_finalize(zckCtx *zck) { zck->index_string = index; zck->index_size = index_size; - /* Rebuild header with index hash set to zeros */ - if(zck->index_digest) { - free(zck->index_digest); - zck->index_digest = NULL; + /* Rebuild header without index hash */ + if(zck->header_digest) { + free(zck->header_digest); + zck->header_digest = NULL; } if(!zck_header_create(zck)) return False; + /* Rebuild signatures */ + if(!zck_sig_create(zck)) + return False; + /* Calculate hash of header */ if(!zck_hash_init(&index_hash, &(zck->hash_type))) { free(index); @@ -113,8 +117,12 @@ int zck_index_finalize(zckCtx *zck) { free(index); return False; } - zck->index_digest = zck_hash_finalize(&index_hash); - if(zck->index_digest == NULL) { + if(!zck_hash_update(&index_hash, zck->sig_string, zck->sig_size)) { + free(index); + return False; + } + zck->header_digest = zck_hash_finalize(&index_hash); + if(zck->header_digest == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to calculate %s checksum for index\n", zck_hash_name_from_type(zck->hash_type.type)); diff --git a/src/lib/index/index_read.c b/src/lib/index/index_read.c index 31d7da0..d50b337 100644 --- a/src/lib/index/index_read.c +++ b/src/lib/index/index_read.c @@ -33,33 +33,11 @@ #include "zck_private.h" int zck_index_read(zckCtx *zck, char *data, size_t size) { - zckHash index_hash = {0}; - char *digest = NULL; size_t length = 0; - /* Check that index checksum matches stored checksum */ - zck_log(ZCK_LOG_DEBUG, "Calculating index checksum\n"); - if(!zck_hash_init(&index_hash, &(zck->hash_type))) + /* Add index to checksum */ + if(!zck_hash_update(&(zck->check_full_hash), data, size)) return False; - if(!zck_hash_update(&index_hash, zck->header_string, zck->header_size)) - return False; - if(!zck_hash_update(&index_hash, data, size)) - return False; - digest = zck_hash_finalize(&index_hash); - if(digest == NULL) { - zck_log(ZCK_LOG_ERROR, - "Unable to calculate %s checksum for header\n", - zck_hash_name_from_type(zck->hash_type.type)); - return False; - } - zck_log(ZCK_LOG_DEBUG, "Checking index checksum\n"); - if(memcmp(digest, zck->index_digest, zck->hash_type.digest_size) != 0) { - free(digest); - zck_log(ZCK_LOG_ERROR, "Index fails checksum test\n"); - return False; - } - zck_log(ZCK_LOG_DEBUG, "Checksum is valid\n"); - free(digest); /* Make sure there's at least enough data for full digest and index count */ if(size < zck->hash_type.digest_size + MAX_COMP_SIZE*2) { @@ -69,14 +47,14 @@ int zck_index_read(zckCtx *zck, char *data, size_t size) { /* Read and configure hash type */ int hash_type; - if(!zck_compint_to_int(&hash_type, data + length, &length)) + if(!compint_to_int(&hash_type, data + length, &length)) return False; if(!zck_set_ioption(zck, ZCK_HASH_CHUNK_TYPE, hash_type)) return False; /* Read number of index entries */ size_t index_count; - if(!zck_compint_to_size(&index_count, data + length, &length)) + if(!compint_to_size(&index_count, data + length, &length)) return False; zck->index.count = index_count; @@ -113,14 +91,14 @@ int zck_index_read(zckCtx *zck, char *data, size_t size) { /* Read and store entry length */ size_t chunk_length = 0; - if(!zck_compint_to_size(&chunk_length, data+length, &length)) + if(!compint_to_size(&chunk_length, data+length, &length)) return False; new->start = idx_loc; new->comp_length = chunk_length; /* Read and store uncompressed entry length */ chunk_length = 0; - if(!zck_compint_to_size(&chunk_length, data+length, &length)) + if(!compint_to_size(&chunk_length, data+length, &length)) return False; new->length = chunk_length; diff --git a/src/lib/io.c b/src/lib/io.c index 488ee19..f841cf9 100644 --- a/src/lib/io.c +++ b/src/lib/io.c @@ -33,6 +33,57 @@ #include "zck_private.h" +int read_header_unread(zckCtx *zck, size_t length) { + if(zck->hdr_buf_size < length) { + zck_log(ZCK_LOG_ERROR, + "Attempting to unread %lu bytes while only %lu were read\n", + length, zck->hdr_buf_size); + return False; + } + zck->hdr_buf_size -= length; + return True; +} + +ssize_t read_header(zckCtx *zck, char **data, size_t length) { + while(zck->hdr_buf_size + length > zck->hdr_buf_read) { + zck->hdr_buf = realloc(zck->hdr_buf, zck->hdr_buf_size + length); + if(zck->hdr_buf == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + zck->hdr_buf_size + length); + return -1; + } + ssize_t rd = read_data(zck->fd, zck->hdr_buf + zck->hdr_buf_read, + zck->hdr_buf_size + length - zck->hdr_buf_read); + if(rd < 0) + return -1; + zck->hdr_buf_read = zck->hdr_buf_read + rd; + length = zck->hdr_buf_read - zck->hdr_buf_size; + } + *data = zck->hdr_buf + zck->hdr_buf_size; + zck->hdr_buf_size += length; + return length; +} + +int close_read_header(zckCtx *zck) { + if(zck->hdr_buf_read > zck->hdr_buf_size) { + zck->comp.data = zmalloc(zck->hdr_buf_read - zck->hdr_buf_size); + if(zck->comp.data == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + zck->hdr_buf_read - zck->hdr_buf_size); + return False; + } + memcpy(zck->comp.data, zck->hdr_buf + zck->hdr_buf_size, + zck->hdr_buf_read - zck->hdr_buf_size); + zck->comp.data_size = zck->hdr_buf_read - zck->hdr_buf_size; + zck->comp.data_loc = zck->comp.data_size; + } + free(zck->hdr_buf); + zck->hdr_buf = NULL; + zck->hdr_buf_read = 0; + zck->hdr_buf_size = 0; + return True; +} + ssize_t read_data(int fd, char *data, size_t length) { if(length == 0) return 0; @@ -69,7 +120,7 @@ int write_data(int fd, const char *data, size_t length) { int write_comp_size(int fd, size_t val) { char data[sizeof(size_t)*2] = {0}; size_t length = 0; - zck_compint_from_size(data, val, &length); + compint_from_size(data, val, &length); return write_data(fd, data, length); } @@ -83,7 +134,7 @@ int read_comp_size(int fd, size_t *val, size_t *length) { *val = 0; return False; } - return !zck_compint_to_size(val, data, length); + return !compint_to_size(val, data, length); } int seek_data(int fd, off_t offset, int whence) { diff --git a/src/lib/zck.c b/src/lib/zck.c index f9358ab..67f5930 100644 --- a/src/lib/zck.c +++ b/src/lib/zck.c @@ -95,6 +95,9 @@ int PUBLIC zck_close(zckCtx *zck) { zck_log(ZCK_LOG_DEBUG, "Writing index\n"); if(!zck_write_index(zck)) return False; + zck_log(ZCK_LOG_DEBUG, "Writing signatures\n"); + if(!zck_write_sigs(zck)) + return False; zck_log(ZCK_LOG_DEBUG, "Writing chunks\n"); if(!chunks_from_temp(zck)) return False; @@ -105,7 +108,11 @@ int PUBLIC zck_close(zckCtx *zck) { close(zck->temp_fd); zck->temp_fd = 0; } + } else { + if(!zck_validate_file(zck)) + return False; } + return True; } @@ -132,9 +139,9 @@ void zck_clear(zckCtx *zck) { free(zck->full_hash_digest); zck->full_hash_digest = NULL; } - if(zck->index_digest) { - free(zck->index_digest); - zck->index_digest = NULL; + if(zck->header_digest) { + free(zck->header_digest); + zck->header_digest = NULL; } if(zck->temp_fd) { close(zck->temp_fd); @@ -175,8 +182,10 @@ zckCtx PUBLIC *zck_init_read (int src_fd) { if(zck == NULL) return NULL; - if(!zck_read_header(zck)) + if(!zck_read_header(zck)) { + zck_free(&zck); return NULL; + } return zck; } @@ -246,7 +255,7 @@ char *get_digest_string(const char *digest, int size) { char PUBLIC *zck_get_header_digest(zckCtx *zck) { if(zck == NULL) return NULL; - return get_digest_string(zck->index_digest, zck->hash_type.digest_size); + return get_digest_string(zck->header_digest, zck->hash_type.digest_size); } char PUBLIC *zck_get_data_digest(zckCtx *zck) { @@ -264,7 +273,7 @@ char PUBLIC *zck_get_chunk_digest(zckIndexItem *item) { ssize_t PUBLIC zck_get_header_length(zckCtx *zck) { if(zck == NULL) return -1; - return zck->header_size + zck->index_size; + return zck->data_offset; } ssize_t zck_get_data_length(zckCtx *zck) { @@ -317,6 +326,7 @@ int zck_import_dict(zckCtx *zck) { if(size == 0) return True; + zck_log(ZCK_LOG_DEBUG, "Reading compression dict\n"); char *data = zmalloc(size); if(data == NULL) { zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", size); @@ -423,7 +433,7 @@ int zck_validate_file(zckCtx *zck) { return False; } zck_log(ZCK_LOG_DEBUG, "Checking data checksum\n"); - zck_log(ZCK_LOG_INFO, "Excpected data checksum: "); + zck_log(ZCK_LOG_INFO, "Expected data checksum: "); for(int i=0; ihash_type.digest_size; i++) zck_log(ZCK_LOG_INFO, "%02x", (unsigned char)zck->full_hash_digest[i]); zck_log(ZCK_LOG_INFO, "\n"); @@ -440,3 +450,34 @@ int zck_validate_file(zckCtx *zck) { free(digest); return True; } + +int zck_validate_header(zckCtx *zck) { + VALIDATE(zck); + char *digest = zck_hash_finalize(&(zck->check_full_hash)); + if(digest == NULL) { + zck_log(ZCK_LOG_ERROR, + "Unable to calculate %s checksum for header\n"); + return False; + } + zck_log(ZCK_LOG_DEBUG, "Checking header checksum\n"); + zck_log(ZCK_LOG_INFO, "Expected header checksum: "); + for(int i=0; ihash_type.digest_size; i++) + zck_log(ZCK_LOG_INFO, "%02x", (unsigned char)zck->header_digest[i]); + zck_log(ZCK_LOG_INFO, "\n"); + zck_log(ZCK_LOG_INFO, "Calculated header checksum: "); + for(int i=0; ihash_type.digest_size; i++) + zck_log(ZCK_LOG_INFO, "%02x", (unsigned char)digest[i]); + zck_log(ZCK_LOG_INFO, "\n"); + if(memcmp(digest, zck->header_digest, zck->hash_type.digest_size) != 0) { + free(digest); + zck_log(ZCK_LOG_ERROR, "Header checksum failed!\n"); + return False; + } + zck_log(ZCK_LOG_DEBUG, "Header checksum valid\n"); + free(digest); + + if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type))) + return False; + + return True; +} diff --git a/src/lib/zck_private.h b/src/lib/zck_private.h index f7979cf..f12fd01 100644 --- a/src/lib/zck_private.h +++ b/src/lib/zck_private.h @@ -42,7 +42,6 @@ typedef struct zckHash { void *ctx; } zckHash; -/*typedef struct zckIndexItem zckIndexItem;*/ typedef void CURL; typedef struct zckMP { @@ -97,27 +96,51 @@ typedef struct zckComp { fcclose close; } zckComp; +typedef struct zckSig { + zckHashType hash_type; + size_t length; + char *signature; + void *ctx; +} zckSig; + +typedef struct zckSigCollection { + int count; + zckSig *sig; +} zckSigCollection; + typedef struct zckCtx { int temp_fd; int fd; int mode; char *full_hash_digest; + char *header_digest; + char *hdr_buf; + size_t hdr_buf_size; + size_t hdr_buf_read; char *header_string; size_t header_size; + char *sig_string; + size_t sig_size; char *index_string; size_t index_size; + size_t data_offset; zckIndex index; zckIndexItem *work_index_item; zckHash work_index_hash; + size_t stream; + int has_streams; + + char *read_buf; + size_t read_buf_size; - char *index_digest; zckHash full_hash; zckHash check_full_hash; zckHash check_chunk_hash; zckComp comp; zckHashType hash_type; zckHashType chunk_hash_type; + zckSigCollection sigs; char *data; size_t data_size; @@ -133,6 +156,8 @@ int zck_validate_file(zckCtx *zck) __attribute__ ((warn_unused_result)); int zck_validate_current_chunk(zckCtx *zck) __attribute__ ((warn_unused_result)); +int zck_validate_header(zckCtx *zck) + __attribute__ ((warn_unused_result)); void zck_clear_work_index(zckCtx *zck); char *get_digest_string(const char *digest, int size) __attribute__ ((warn_unused_result)); @@ -190,6 +215,12 @@ int read_comp_size(int fd, size_t *val, size_t *length) __attribute__ ((warn_unused_result)); int chunks_from_temp(zckCtx *zck) __attribute__ ((warn_unused_result)); +ssize_t read_header(zckCtx *zck, char **data, size_t length) + __attribute__ ((warn_unused_result)); +int read_header_unread(zckCtx *zck, size_t length) + __attribute__ ((warn_unused_result)); +int close_read_header(zckCtx *zck) + __attribute__ ((warn_unused_result)); /* header.c */ int zck_read_initial(zckCtx *zck) @@ -198,14 +229,22 @@ int zck_read_header_hash(zckCtx *zck) __attribute__ ((warn_unused_result)); int zck_read_ct_is(zckCtx *zck) __attribute__ ((warn_unused_result)); +int zck_header_hash(zckCtx *zck) + __attribute__ ((warn_unused_result)); int zck_read_index(zckCtx *zck) __attribute__ ((warn_unused_result)); +int zck_read_sig(zckCtx *zck) + __attribute__ ((warn_unused_result)); int zck_read_header(zckCtx *zck) __attribute__ ((warn_unused_result)); int zck_header_create(zckCtx *zck) __attribute__ ((warn_unused_result)); +int zck_sig_create(zckCtx *zck) + __attribute__ ((warn_unused_result)); int zck_write_header(zckCtx *zck) __attribute__ ((warn_unused_result)); +int zck_write_sigs(zckCtx *zck) + __attribute__ ((warn_unused_result)); /* comp/comp.c */ int zck_comp_add_to_dc(zckComp *comp, const char *src, size_t src_size) @@ -240,12 +279,12 @@ int zck_dl_range_chk_chunk(zckDL *dl, char *url, int is_chunk) __attribute__ ((warn_unused_result)); /* compint.c */ -int zck_compint_from_int(char *compint, int val, size_t *length) +int compint_from_int(char *compint, int val, size_t *length) __attribute__ ((warn_unused_result)); -void zck_compint_from_size(char *compint, size_t val, size_t *length); -int zck_compint_to_int(int *val, const char *compint, size_t *length) +void compint_from_size(char *compint, size_t val, size_t *length); +int compint_to_int(int *val, const char *compint, size_t *length) __attribute__ ((warn_unused_result)); -int zck_compint_to_size(size_t *val, const char *compint, size_t *length) +int compint_to_size(size_t *val, const char *compint, size_t *length) __attribute__ ((warn_unused_result)); diff --git a/src/unzck.c b/src/unzck.c index 1e61f59..429bfdb 100644 --- a/src/unzck.c +++ b/src/unzck.c @@ -64,36 +64,37 @@ int main (int argc, char *argv[]) { exit(1); } + int good_exit = False; + zckCtx *zck = zck_init_read(src_fd); if(zck == NULL) - exit(1); + goto error1; char *data = malloc(BLK_SIZE); - int good_exit = False; while(True) { ssize_t read = zck_read(zck, data, BLK_SIZE); if(read < 0) - goto error; + goto error2; if(read == 0) break; - if(read > BLK_SIZE) - printf("read: %lu\n", (long unsigned)read); if(write(dst_fd, data, read) != read) { printf("Error writing to %s\n", out_name); - goto error; + goto error2; } } if(!zck_close(zck)) - goto error; + goto error2; + good_exit = True; -error: +error2: free(data); + zck_free(&zck); +error1: if(!good_exit) unlink(out_name); free(out_name); close(src_fd); close(dst_fd); - zck_free(&zck); if(!good_exit) exit(1); exit(0); diff --git a/src/zck_delta_size.c b/src/zck_delta_size.c index dee1898..d56c9f4 100644 --- a/src/zck_delta_size.c +++ b/src/zck_delta_size.c @@ -48,11 +48,10 @@ int main (int argc, char *argv[]) { } zckCtx *zck_src = zck_init_read(src_fd); if(zck_src == NULL) { - printf("Unable to open %s\n", argv[1]); + printf("Unable to read header from %s\n", argv[1]); exit(1); } - if(!zck_close(zck_src)) - exit(1); + close(src_fd); int tgt_fd = open(argv[2], O_RDONLY); if(tgt_fd < 0) { @@ -65,8 +64,7 @@ int main (int argc, char *argv[]) { printf("Unable to open %s\n", argv[2]); exit(1); } - if(!zck_close(zck_tgt)) - exit(1); + close(tgt_fd); if(zck_get_chunk_hash_type(zck_tgt) != zck_get_chunk_hash_type(zck_src)) { printf("ERROR: Chunk hash types don't match:\n"); diff --git a/src/zck_read_header.c b/src/zck_read_header.c index 662852e..1dcb460 100644 --- a/src/zck_read_header.c +++ b/src/zck_read_header.c @@ -49,11 +49,9 @@ int main (int argc, char *argv[]) { } zckCtx *zck = zck_init_read(src_fd); if(zck == NULL) { - perror("Unable to read header\n"); + printf("Unable to read header\n"); exit(1); } - if(!zck_close(zck)) - exit(1); close(src_fd); printf("Overall checksum type: %s\n", zck_hash_name_from_type(zck_get_full_hash_type(zck)));